#include "tokenizer/qwen2_tokenizer.h"

#include "uninorms.h"

#include <codecvt>
#include <locale>

namespace pa
{
Qwen2Tokenizer::Qwen2Tokenizer() {}
Qwen2Tokenizer::~Qwen2Tokenizer() {}

std::u32string Qwen2Tokenizer::unicode_nfc_normalize(const std::string& prompt_origin)
{
    std::u32string prompt_utf32 = std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t>{}.from_bytes(prompt_origin);
    unilib::uninorms::nfc(prompt_utf32);
    return prompt_utf32;
}

} // namespace pa